
import requests
import MySQLdb
from MySQLdb import OperationalError


# 配置数据库
db_config = {
    'host': 'localhost',
    'user': 'root',  # 替换为你的用户名
    'password': '',  # 替换为你的密码
    'database': 'bilibili',  # 替换为你的数据库名
    'port': 3307,
    'autocommit': True  # 自动提交事务
}

url = "https://careers.tencent.com/tencentcareer/api/post/Query"
params = {
    "countryId": "",
    "cityId": "",
    "bgIds": "",
    "productId": "",
    "categoryId": "",
    "parentCategoryId": "",
    "attrId": "",
    "keyword": "",
    "pageIndex": "1",
    "pageSize": "10",
    "language": "zh-cn",
    "area": "cn"
}

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0"
}

try:
    # 建立数据库连接
    conn = MySQLdb.connect(**db_config)
    cursor = conn.cursor()
    print("数据库连接成功")
    max_batch_size = 10000
    cache_data_list = []
    # 数据的爬取
    for page in range(1, 239):
        params["pageIndex"] = page
        response = requests.get(url, headers=headers, params=params)
        if response.status_code == 200:
            items = response.json().get("Data").get("Posts")
            # 处理数据并验证
            for item in items:
                # 提前初始化为默认值
                name = item.get("RecruitPostName", "")
                address = item.get("LocationName", "")
                work_experience = item.get("RequireWorkYearsName", "")
                job = item.get("Responsibility")

                # 添加到缓存列表中
                cache_data_list.append([name, address, work_experience, job])

        print(f"第 {page} 页数据处理完成，累计 {len(cache_data_list)} 条")

    # 批量插入数据库
    if cache_data_list:
        sql = """
               INSERT INTO tb_zhaopin (name, address, work_experience, job)
               VALUES (%s, %s, %s, %s)
               """
        cursor.executemany(sql, cache_data_list)
        conn.commit()
        print(f"成功插入 {len(cache_data_list)} 条数据")
    else:
        print(f"没有可插入的数据")
except OperationalError as e:
    print(f"数据库操作错误: {str(e)}")
except Exception as e:
    print(f"发生未知错误: {str(e)}")
finally:
    # 关闭资源
    if conn:
        cursor.close()
        conn.close()
        print("数据库连接已关闭")
